import requests
from config import config
import time
import re
import test
from data_save import save_mysql

# 获取招聘数据源码
def get_data(url,headers):
    response = requests.get(url, headers=headers)
    if response.status_code == 200:
        return response.text
    else:
        print(response.status_code)

# 解析详细招聘数据
def parse_job_url(html):
    pattern = re.compile(r'"jobName":"(.*?)".*?"cityString":"(.*?)".*?"degreeString":"(.*?)".*?"fullCompanyName":"(.*?)".*?"jobDescribe":"(.*?)".*?"jobSalaryMax":"(.*?)".*?"jobSalaryMin":"(.*?)"', re.S)
    items = re.findall(pattern, html)
    return items

def main():
    for item in config.job_list:
        #  调用函数，创建数据表
        # save_mysql.create_table(item)
        #  开始抓取数据并保存在数据库中
        for page in range(50):
            time.sleep(10)
            print("正在抓取"+item+"的第 "+str(page+1)+" 页")
            job_url = config.job_url(page,item)
            headers = config.job_headers
            html = get_data(job_url,headers)
            job_datail = parse_job_url(html)
            for job in job_datail:
                try:
                    save_mysql.insert_data(item,job)
                except Exception as e:
                    print("数据存储异常")

if __name__ == '__main__':
    main()
